{
    "cells": [
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "%matplotlib inline\n",
                "from ipywidgets import interactive\n",
                "import ipywidgets as widgets\n",
                "from ipywidgets import fixed\n",
                "\n",
                "import numpy as np\n",
                "import scipy.special\n",
                "import matplotlib.pyplot as plt\n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# RBF Demo\n",
                "\n",
                "In this notebook, you will implement a finite-dimensional approximation to the RBF kernel and perform kernel ridge regression on some test data. You should compare the behavior of your approximation to the true model and comment on your observations.\n",
                "\n",
                "First, fill in your approximation to the true RBF kernel below.\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "def make_phi(sigma, d):\n",
                "    def phi(x):\n",
                "        x = x.reshape((-1,))\n",
                "        n = len(x)\n",
                "        out = np.zeros((d, n)) # each column is a separate featurized data point\n",
                "        # fill in the values of the d-dimensional `out` feature vector\n",
                "        ### start approximation ###\n",
                "\n",
                "        ### end approximation ###\n",
                "\n",
                "        return out\n",
                "\n",
                "    return phi\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "from sklearn.linear_model import Ridge\n",
                "\n",
                "def ridge_regression(x, y, lamb, phi):\n",
                "    X = phi(x)\n",
                "    return Ridge(lamb, fit_intercept=False).fit(X.T, y).coef_\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "def generate_x(n, x_type, x_low=-1, x_high=1):\n",
                "    if x_type == 'grid':\n",
                "        x = np.linspace(x_low, x_high, n, endpoint = False).astype(np.float64)\n",
                "\n",
                "    elif x_type == 'uniform_random':\n",
                "        x = np.sort(np.random.uniform(x_low, x_high, n).astype(np.float64))\n",
                "        #Note that for making it easy for plotting we sort the randomly sampled x in ascending order\n",
                "    else:\n",
                "        raise ValueError\n",
                "\n",
                "\n",
                "    return x\n",
                "\n",
                "\n",
                "test_x = generate_x(100, 'uniform_random')\n",
                "\n",
                "def plot_estimate(n_train, d, sigma, lambda_param):\n",
                "    x = generate_x(n_train, 'uniform_random')\n",
                "    y = np.sin(x * 10)\n",
                "    phi = make_phi(sigma, d)\n",
                "    plt.scatter(x, y, label=\"Training points\")\n",
                "    m = ridge_regression(x, y, lambda_param, phi)\n",
                "    plt.plot(test_x, phi(test_x).T @ m, label=\"Kernel fit\")\n",
                "    plt.scatter(test_x, phi(test_x).T @ m, label=\"Kernel fit (test points)\")\n",
                "    print(\"Kernel ridge regression fit\")\n",
                "    plt.legend(loc='lower right')\n",
                "    plt.show()\n",
                "\n",
                "    y2 = np.zeros(y.shape)\n",
                "    y2[len(y2) // 2] = 1\n",
                "    plt.scatter(x, y2, label=\"Training points\")\n",
                "    m = ridge_regression(x, y2, lambda_param, phi)\n",
                "    plt.plot(test_x, phi(test_x).T @ m, label=\"Kernel fit (curve)\")\n",
                "    print(\"Impulse response\")\n",
                "    plt.legend(loc='upper right')\n",
                "    plt.show()\n",
                "\n",
                "\n",
                "n_train = widgets.IntSlider(\n",
                "    value=30,\n",
                "    min=10,\n",
                "    max=99,\n",
                "    step=1,\n",
                "    description='$n_{train}$',\n",
                "    continuous_update=False\n",
                ")\n",
                "d = widgets.IntSlider(\n",
                "    value=99,\n",
                "    min=0,\n",
                "    max=99,\n",
                "    step=1,\n",
                "    description='$d$',\n",
                "    continuous_update=False\n",
                ")\n",
                "sigma = widgets.FloatSlider(\n",
                "    value=0.02,\n",
                "    min=0.02,\n",
                "    max=0.5,\n",
                "    step=0.02,\n",
                "    description='$\\sigma$',\n",
                "    continuous_update=False\n",
                ")\n",
                "lambda_param = widgets.FloatSlider(\n",
                "    value=0.02,\n",
                "    min=0.02,\n",
                "    max=1,\n",
                "    step=0.02,\n",
                "    description='$\\lambda$',\n",
                "    continuous_update=False\n",
                ")\n",
                "\n",
                "interactive(plot_estimate, n_train=n_train, d=d, sigma=sigma, lambda_param=lambda_param)\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": []
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "Python 3",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.6.5"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 2
}